 # -*- coding: utf-8 -*-
# Copyright Š 2010 alexliyu
#
# This file is part of cntop-news.
#
# cntop-news is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cntop-news is base on Top-news-buzz ,
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cntop-news.  If not, see <http://www.gnu.org/licenses/>.



import logging
from BeautifulSoup import BeautifulSoup
from BTSelector import findAll
from time import sleep
from google.appengine.api import urlfetch
from google.appengine.api.urlfetch_errors import DownloadError
import exceptions


def parsehtml(html_content, feed_url, feed_target):
#    pagehtml = unicode( html_content,'gb2312','ignore').encode('utf-8','ignore')
    pagehtml=html_content
    target=feed_target
    ret=''
    if isinstance(pagehtml, unicode):
#        pagehtml.encode('utf-8')
         pass
    else:
        pagehtml=encoding(pagehtml)
#        pagehtml=pagehtml.encode('utf-8')

    try:
            #解析元数据和url

        soup = BeautifulSoup(pagehtml)
        soup.prettify()
        try:
            tret=findAll(target,soup)
            ret =str(tret[0])
          #ret=encoding(ret)
            #ret=ret.encode('utf-8')

            if len(ret)==0:
                logging.error('The feed %s `s target %s is bad target', feed_url, target)
                return None
            else:
#                logging.info(ret)
                #ret=encoding(ret)

                logging.info('add the new article now,the new one len to %s,type to %s', len(ret), type(ret))
                return ret
        except  Exception,data:
            logging.error('something is wrong at url %s ,the target is %s ', feed_url, data)
            return None

    except Exception:
        logging.error('Could not parse this,the html has misstake is %s', feed_url)
        return None
#    #休眠2.5秒
#    sleep(2.5)

def encoding(s):
    cl = ['utf-8', 'gb2312', 'gbk']
    ce=''
    for a in cl:
        try:
            ce=s.decode(a)
            break
        except UnicodeDecodeError:
            pass
    logging.info(a)
    return ce
